{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 节点对象"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 基类节点对象\n",
    "- `inputs` 属性，输入节点组成的 `list`\n",
    "- `outputs` 属性，输出节点组成的 `list`\n",
    "- `forward` 方法，正向传播，通过输入节点计算输出\n",
    "- `backward` 方法，反向传播计算梯度\n",
    "- `gradients` 属性，保存当前节点对输入节点的梯度，为 `dict` 对象\n",
    "- `value` 属性，节点操作后的输出值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Node:\n",
    "    def __init__(self, inputs=[]):\n",
    "        self.inputs = inputs\n",
    "        self.value = None\n",
    "        self.outputs = []\n",
    "        self.gradients = {}\n",
    "        \n",
    "        for node in self.inputs:\n",
    "            # 将当前节点添加到，它的所有输入节点的输出节点中\n",
    "            node.outputs.append(self) # build a connection relationship\n",
    "            \n",
    "    def forward(self):\n",
    "        raise NotImplemented\n",
    "    \n",
    "    def backward(self):\n",
    "        raise NotImplemented"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 接受训练数据及参数的`Input`节点 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Input(Node):\n",
    "    \"\"\"\n",
    "    用来接受训练数据，其输入节点为空\n",
    "    \"\"\"\n",
    "    def __init__(self, name=''):\n",
    "        Node.__init__(self, inputs=[])\n",
    "        self.name = name\n",
    "    \n",
    "    def forward(self, value=None):\n",
    "        if value is not None:\n",
    "            self.value = value\n",
    "    \n",
    "    def backward(self):\n",
    "        self.gradients = {}\n",
    "        \n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self] = grad_cost\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return \"Input Node: {}\".format(self.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 仿射节点 \n",
    "- 输入节点`X,W,B`\n",
    "- 执行操作`X*W+B`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([1,2,3])\n",
    "W = np.array([[1,2],[2,4],[3,5]])\n",
    "B = np.array([1,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([15, 28])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(X,W)+B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Node):\n",
    "    def __init__(self, nodes, weights, bias):\n",
    "        self.w_node = weights\n",
    "        self.x_node = nodes\n",
    "        self.b_node = bias\n",
    "        Node.__init__(self, inputs=[nodes, weights, bias])\n",
    "        \n",
    "    def forward(self):\n",
    "        self.value = np.dot(self.x_node.value, self.w_node.value)+self.b_node.value\n",
    "        \n",
    "    def backward(self):\n",
    "        for node in self.outputs:\n",
    "            grad_cost = node.gradients[self]\n",
    "            self.gradients[self.w_node] = np.dot(self.x_node.value.T, grad_cost)\n",
    "            self.gradients[self.b_node] = np.sum(grad_cost*1, axis=0, keepdims=False)\n",
    "            self.gradients[self.x_node] = np.dot(grad_cost, self.w_node.value.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 激活节点 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid(Node):\n",
    "    def __init__(self, node):\n",
    "        Node.__init__(self, [node])\n",
    "        self.x_node = node\n",
    "    \n",
    "    def _sigmoid(self, x):\n",
    "        return 1./(1+np.exp(-1*x))\n",
    "    \n",
    "    def forward(self):\n",
    "        self.value = self._sigmoid(self.x_node.value)\n",
    "        \n",
    "    def backward(self):\n",
    "        y = self.value\n",
    "        self.partial = y*(1-y)\n",
    "        for n in self.outputs:\n",
    "            grad_cost = n.gradients[self]\n",
    "            self.gradients[self.x_node] = grad_cost * self.partial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 损失函数节点 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MSE(Node):\n",
    "    def __init__(self, y_true, y_hat):\n",
    "        self.y_true_node = y_true\n",
    "        self.y_hat_node = y_hat\n",
    "        Node.__init__(self, inputs=[y_true,y_hat])\n",
    "        \n",
    "    def forward(self):\n",
    "        y_true_flatten = self.y_true_node.value.reshape(-1,1)\n",
    "        y_hat_flatten = self.y_hat_node.value.reshape(-1,1)\n",
    "        self.diff = y_true_flatten-y_hat_flatten\n",
    "        self.value = np.mean(self.diff**2)\n",
    "        \n",
    "    def backward(self):\n",
    "        n = self.y_hat_node.value.shape[0]\n",
    "        \n",
    "        self.gradients[self.y_true_node] = (2/n)*self.diff\n",
    "        self.gradients[self.y_hat_node] = (-2/n)*self.diff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 创建图"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 拓扑排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topological_sort(data_with_value):\n",
    "    \"\"\"\n",
    "    输入训练数据，Input对象，创建图；\n",
    "    \"\"\"\n",
    "    feed_dict = data_with_value\n",
    "    input_nodes = [n for n in feed_dict.keys()]\n",
    "    \n",
    "    # 每个节点的输入节点与输出节点\n",
    "    # 生成图\n",
    "    G = {}\n",
    "    nodes = [n for n in input_nodes]\n",
    "    while len(nodes)>0:\n",
    "        n = nodes.pop(0)\n",
    "        if n not in G:\n",
    "            G[n] = {'in':set(),'out':set()}\n",
    "        for m in n.outputs:\n",
    "            if m not in G:\n",
    "                G[m] = {'in':set(), 'out':set()}\n",
    "            G[n]['out'].add(m)\n",
    "            G[m]['in'].add(n)\n",
    "            nodes.append(m)\n",
    "\n",
    "    # 找出所有只有出度的节点，随机选择一个，并删除该节点所有的输出边；循环至选择出所有节点\n",
    "    L = []\n",
    "    S = set(input_nodes)\n",
    "    while len(S) > 0:\n",
    "        n = S.pop()\n",
    "\n",
    "        if isinstance(n, Input):\n",
    "            n.value = feed_dict[n]\n",
    "            ## if n is Input Node, set n'value as \n",
    "            ## feed_dict[n]\n",
    "            ## else, n's value is caculate as its\n",
    "            ## inbounds\n",
    "\n",
    "        L.append(n)\n",
    "        for m in n.outputs:\n",
    "            G[n]['out'].remove(m)\n",
    "            G[m]['in'].remove(n)\n",
    "            # if no other incoming edges add to S\n",
    "            if len(G[m]['in']) == 0:\n",
    "                S.add(m)\n",
    "    return L"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_one_batch(topological_sorted_graph):\n",
    "    # graph 是经过拓扑排序之后的 一个list\n",
    "    for node in topological_sorted_graph:\n",
    "        node.forward()\n",
    "    for node in topological_sorted_graph[::-1]:\n",
    "        node.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sgd_update(trainable_nodes, learning_rate=1e-2):\n",
    "    for t in trainable_nodes:\n",
    "        t.value += -1 * learning_rate * t.gradients[t]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "\n",
    "data = load_boston()\n",
    "\n",
    "X_ = data['data']\n",
    "y_ = data['target']\n",
    "\n",
    "X_ = (X_-np.mean(X_,axis=0))/np.std(X_,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_features = X_.shape[1]\n",
    "n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1_, b1_ = np.random.randn(n_features, n_hidden), np.zeros(n_hidden)\n",
    "W2_, b2_ = np.random.randn(n_hidden, 1), np.zeros(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建图中节点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = Input(name='X'), Input(name='y')  # tensorflow -> placeholder\n",
    "W1, b1 = Input(name='W1'), Input(name='b1') # tensorflow -> Varaible\n",
    "W2, b2 = Input(name='W2'), Input(name='b2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_output = Linear(X, W1, b1)\n",
    "sigmoid_output = Sigmoid(linear_output)\n",
    "yhat = Linear(sigmoid_output, W2, b2)\n",
    "loss = MSE(y, yhat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$y^1=X_{[13,]}\\cdot W^1_{[13,10]}+b^1_{[10,]}$\n",
    "   \n",
    "$y^{\\sigma} = \\sigma(y^1_{[10,]})$\n",
    "   \n",
    "$\\hat{y}= y^{\\sigma}_{[10,]}\\cdot W^2_{[10,1]}+b^2_{[1,]}$  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_with_value = {  # -> feed_dict \n",
    "    X: X_, \n",
    "    y: y_, \n",
    "    W1: W1_, \n",
    "    W2: W2_, \n",
    "    b1: b1_, \n",
    "    b2: b2_\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph = topological_sort(input_node_with_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Input Node: y,\n",
       " Input Node: b1,\n",
       " Input Node: b2,\n",
       " Input Node: W2,\n",
       " Input Node: W1,\n",
       " Input Node: X,\n",
       " <__main__.Linear at 0x1c29f9f5c08>,\n",
       " <__main__.Sigmoid at 0x1c29f9f5488>,\n",
       " <__main__.Linear at 0x1c29fa2fe08>,\n",
       " <__main__.MSE at 0x1c29fa2f888>]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import resample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1, loss = 656.861\n",
      "Epoch: 101, loss = 30.534\n",
      "Epoch: 201, loss = 22.998\n",
      "Epoch: 301, loss = 19.092\n",
      "Epoch: 401, loss = 21.147\n",
      "Epoch: 501, loss = 16.537\n",
      "Epoch: 601, loss = 17.207\n",
      "Epoch: 701, loss = 13.458\n",
      "Epoch: 801, loss = 12.301\n",
      "Epoch: 901, loss = 12.968\n",
      "Epoch: 1001, loss = 11.508\n",
      "Epoch: 1101, loss = 14.529\n",
      "Epoch: 1201, loss = 9.997\n",
      "Epoch: 1301, loss = 10.658\n",
      "Epoch: 1401, loss = 9.573\n",
      "Epoch: 1501, loss = 8.555\n",
      "Epoch: 1601, loss = 8.751\n",
      "Epoch: 1701, loss = 8.240\n",
      "Epoch: 1801, loss = 9.727\n",
      "Epoch: 1901, loss = 6.514\n",
      "Epoch: 2001, loss = 9.124\n",
      "Epoch: 2101, loss = 7.923\n",
      "Epoch: 2201, loss = 7.493\n",
      "Epoch: 2301, loss = 6.686\n",
      "Epoch: 2401, loss = 7.282\n",
      "Epoch: 2501, loss = 7.931\n",
      "Epoch: 2601, loss = 5.882\n",
      "Epoch: 2701, loss = 6.295\n",
      "Epoch: 2801, loss = 7.290\n",
      "Epoch: 2901, loss = 8.225\n",
      "Epoch: 3001, loss = 5.995\n",
      "Epoch: 3101, loss = 6.551\n",
      "Epoch: 3201, loss = 5.586\n",
      "Epoch: 3301, loss = 6.341\n",
      "Epoch: 3401, loss = 8.904\n",
      "Epoch: 3501, loss = 5.612\n",
      "Epoch: 3601, loss = 8.401\n",
      "Epoch: 3701, loss = 7.765\n",
      "Epoch: 3801, loss = 7.036\n",
      "Epoch: 3901, loss = 7.092\n",
      "Epoch: 4001, loss = 6.423\n",
      "Epoch: 4101, loss = 6.847\n",
      "Epoch: 4201, loss = 7.679\n",
      "Epoch: 4301, loss = 5.894\n",
      "Epoch: 4401, loss = 8.830\n",
      "Epoch: 4501, loss = 5.613\n",
      "Epoch: 4601, loss = 5.893\n",
      "Epoch: 4701, loss = 5.896\n",
      "Epoch: 4801, loss = 5.952\n",
      "Epoch: 4901, loss = 5.773\n"
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "epochs = 5000\n",
    "\n",
    "batch_size = 64\n",
    "\n",
    "steps_per_epoch = X_.shape[0] // batch_size\n",
    "\n",
    "for i in range(epochs):\n",
    "    loss = 0\n",
    "\n",
    "    for batch in range(steps_per_epoch):\n",
    "        #indices = np.random.choice(range(X_.shape[0]), size=10, replace=True)\n",
    "        #X_batch = X_[indices]\n",
    "        #y_batch = y_[indices]\n",
    "        X_batch, y_batch = resample(X_, y_, n_samples=batch_size)\n",
    "\n",
    "        X.value = X_batch\n",
    "        y.value = y_batch\n",
    "\n",
    "        input_node_with_value = {  # -> feed_dict \n",
    "            X: X_batch,\n",
    "            y: y_batch,\n",
    "            W1: W1.value,\n",
    "            W2: W2.value,\n",
    "            b1: b1.value,\n",
    "            b2: b2.value,\n",
    "        }\n",
    "\n",
    "        graph = topological_sort(input_node_with_value)\n",
    "\n",
    "        training_one_batch(graph)\n",
    "\n",
    "        learning_rate = 1e-3\n",
    "\n",
    "        sgd_update(trainable_nodes=[W1, W2, b1, b2],\n",
    "                   learning_rate=learning_rate)\n",
    "\n",
    "        loss += graph[-1].value\n",
    "\n",
    "    if i % 1000 == 0:\n",
    "        print('Epoch: {}, loss = {:.3f}'.format(i + 1, loss / steps_per_epoch))\n",
    "        losses.append(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查看结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1c2a5997ec8>]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD2CAYAAADcUJy6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAV9klEQVR4nO3da4wdZ33H8d9/LufMXrzrtXfjOCaJgZR7iQimpZAKxTSpQAEBLRKKIqLmRd4UIQTiBZUq8QYJXhQJWoQaCSSEQJVoRAVBAkKSIqBctA51KDcRIE6cxPH6utdznX9fzJyzF2/i4zUn633m+5FWZ85zdn3m2bP+zX+eeWbG3F0AgPBE270CAIDhIOABIFAEPAAEioAHgEAR8AAQqGS7V6BnenraDx48uN2rAQA7ypEjR065+8xmr10xAX/w4EHNzs5u92oAwI5iZsee6zWGaAAgUAQ8AASKgAeAQBHwABAoAh4AAkXAA0CgCHgACNSOD/jfnljQv3z3tzq92NzuVQGAK8qOD/g/zC3qXx96THMEPACss+MDvp4WXWi0821eEwC4suz8gE9iSVKz3d3mNQGAK8uOD/isrOCbHSp4AFhrxwd8r4JvUMEDwDoBBDwVPABsZscHfJaWY/AEPACss+MDvlfBM0QDAOsFEPBU8ACwmZ0f8CkVPABsZucHPAdZAWBTOz7gzUz1JFKzQwUPAGvt+ICXiiq+yaUKAGCdMAI+jangAWCDIAI+S6ngAWCjIAK+nsRqUMEDwDqBBDwVPABsFETAZ2nMNEkA2CCIgK8nESc6AcAGwQQ8FTwArBdEwGdpTAUPABsEEfBU8ABwoUACnhOdAGCjIAI+SyM1mCYJAOsMHPBm9mEz+56ZTZvZD8zsF2b2yfK1gdqGhUsVAMCFBgp4M7te0l3l0w9J+pakGyW9zcxedgltQ5GVY/DuPqy3AIAdZ9AK/jOSPlYuH5b0gLvnkr4v6ZZLaFvHzO4xs1kzm52bm9tyJ+ppLHep1WWYBgB6LhrwZnaHpKOSflU27ZV0vlyel7TnEtrWcfd73f2Qux+amZnZah+46QcAbCIZ4Htul3SdpL+V9HJJuaTJ8rVJSccknRqwbSjqaXlf1nYuZcN6FwDYWS5awbv7He5+s6T3SToi6XOSbjOzSNJbJD0s6cEB24aiV8FzshMArNrKNMnPSnq7pEclfcvdH7uEtqFgiAYALjTIEI0kyd0fl/Q35dO/3vDaqUHahiUrh2io4AFgVRAnOlHBA8CFAgn48iArJzsBQF8QAZ+lZQXP5QoAoC+IgKeCB4ALBRHw/QqeMXgA6Asi4OvMogGAC4QR8MyiAYALBBHw2dpLFQAAJAUS8FyqAAAuFETAJ5EpMoZoAGCtIALezJSlMRU8AKwRRMBLxTANFTwArAoo4LkvKwCsFUzAZ2mkBrNoAKAvmICnggeA9YIJ+CxlDB4A1gom4OsJs2gAYK1wAp4KHgDWCSfgk5hLFQDAGuEEfBqpwUFWAOgLJ+CTiAoeANYIJuCzlGmSALBWMAFPBQ8A6wUU8DGzaABgjWACPksjtbq5urlv96oAwBUhmICvJ8VdnVpU8QAgKaCAz9LefVk50AoAUkAB36vguaIkABQCCngqeABYK5iAz9KigmcmDQAUggn4XgXPFSUBoBBOwPcPslLBA4AUUMD3hmio4AGgEEzA9w+yMosGACQFFfAcZAWAtS4a8GaWmNnXzOxHZvZFM8vM7H4zO2pmX7bCQG3D7EjvRCeGaACgMEgF/y5JR939zZL2S/qApOPufqOkKUm3SrpzwLahoYIHgPUGCfhvS/q0mSWSdku6SdID5WsPSbpF0uEB24aGSxUAwHoXDXh3X3T3ZUk/kvSspL2Szpcvz0vacwlt65jZPWY2a2azc3Nzl9MPLlUAABsMMga/18zqkt6kYqjlNZImy5cnJZ0qvwZpW8fd73X3Q+5+aGZm5nL6waUKAGCDQYZoPiLpve7elbQs6ROSbitfOyzpYUkPDtg2NFFkqsURY/AAUBok4D8n6W4z+7Gk05K+IOmAmT0q6YyKIP/KgG1DVU8iZtEAQCm52De4+1MqKvC1bt/wvDlg21DVUyp4AOgJ5kQnqTjQSgUPAIWwAp4KHgD6wgr4JOZaNABQCirgszRimiQAlIIK+HoSUcEDQCmogM/SmAoeAEpBBXwxD54KHgCk4AKeCh4AeoIK+IxpkgDQF1TAc6ITAKwKLOCp4AGgJ6iAz9Kignf37V4VANh2QQV8PYmUu9TJCXgACCvg+7ftY5gGAIIK+Czt3baPA60AEFTAr962jwoeAIIK+F4F36SCB4CwAr5XwXO5AgAILuDLCp7LFQBAYAHPLBoA6Asr4BNm0QBAT2ABTwUPAD1BBTzz4AFgVVABTwUPAKvCCngOsgJAX1ABz4lOALAqqIBniAYAVgUV8LU4khkVPABIgQW8mameRGpQwQNAWAEvFSc7UcEDQIABn6XclxUApAADvp7EnOgEAAoy4KngAUAKMOCzlAoeAKQAA54KHgAKAwW8mX3JzH5iZt8ws3Ezu9/MjprZl62QDdI27M5IxeUKCHgAGCDgzexmSYm7v1HShKS7JR139xslTUm6VdKdA7YNXcZBVgCQNFgF/6ykz6z5/o9LeqB8/pCkWyQdHrBt6KjgAaBw0YB399+5+8/M7N2Sckk/l3S+fHle0h5JewdsW8fM7jGzWTObnZubu6yO9GRJzD1ZAUCDj8G/U9IHJb1D0glJk+VLk5JOlV+DtK3j7ve6+yF3PzQzM7PVPqxTTyM12lTwADDIGPzVkj4q6XZ3X5D0oKTbypcPS3r4EtqGjksVAEBhkAr+Lkn7JX3HzH4oKZV0wMwelXRGRZB/ZcC2oWMMHgAKycW+wd0/JelTG5r/fcPzpqTbB2gbunoSq9nJ5e56gWZmAsAVKcgTnSRu+gEAwQX86m37CHgA1RZcwK9W8BxoBVBtAQc8FTyAagsu4HtDNFyuAEDVBRfwVPAAUAgv4HsHWRmDB1BxwQV8VlbwXK4AQNUFF/BU8ABQCC7gs7Qcg6eCB1BxwQV8PSln0VDBA6i4AAOeCh4ApAADnnnwAFAILuCZBw8ABQIeAAIVXMAncaQkMoZoAFRecAEvFVU8FTyAqgsz4NOYE50AVF6QAZ8lEZcqAFB5QQZ8UcET8ACqLcyATyI1OcgKoOLCDPg0VoMKHkDFhRnwVPAAEGbAZ1TwABBmwFPBA0DAAd+iggdQcUEGfJbGXKoAQOUFGfBcqgAAgg14TnQCgCADPksjhmgAVF6QAV9PYnVyV6dLFQ+guoIM+CwtutUi4AFUWJAB37urE1eUBFBlYQZ8eeNtrgkPoMqCDPjeEA0VPIAqCzLg6wkVPAAMFPBmlprZN8vlzMzuN7OjZvZlKwzUNtyurOqNwTep4AFU2EUD3sxGJB2RdGvZdKek4+5+o6Spsn3QthdEVo7BMxceQJVdNODdfcXdXyvpeNl0WNID5fJDkm65hLZ1zOweM5s1s9m5ubktd2KjfgXP2awAKmwrY/B7JZ0vl+cl7bmEtnXc/V53P+Tuh2ZmZrawKptbHYMn4AFUV7KFnzklabJcniyfjw/Y9oJYnUXDEA2A6tpKBf+gpNvK5cOSHr6EthcEFTwAbC3gvyLpgJk9KumMiiAftO0F0avgmSYJoMoGHqJx9xvKx6ak2ze8PGjbC6JXwXOiE4AqC/NEJyp4AAg04LnYGACEGfBmploSUcEDqLQgA14q78tKBQ+gwoIN+CyNqeABVFqwAU8FD6Dqwg54TnQCUGHBBnyWxlyqAEClBRvwVPAAqi7YgOcgK4CqCzbg60nEiU4AKi3ggKeCB1BtwQZ8llLBA6i2YAOeCh5A1YUb8CmzaABUW7ABzzx4AFUXbMD35sG7+3avCgBsi6AD3l1qdwl4ANUUbMBnaXnbPg60AqioYAO+d1cnrigJoKrCDfiygmeqJICqCjfguS8rgIoLOOCp4AFUW7ABn6VU8ACqLdiAp4IHUHXhBnxZwXO5AgBVFWzAZ70KnssVAKioYAO+V8GfWWpv85oAwPYINuD3jtVUTyL909d/ob///P/oqz99QudXCHsA1WFXysW4Dh065LOzs3/Sf/PE+Yb+63+f0n1Hjut3JxdVSyLd+sp9es9NB3Tzn033D8QCwE5lZkfc/dCmr4Uc8D3urv97al73PXJc3zj6tM4stZREphuuGterrpnQq6+Z1KuvmdCrrpnQeC3RmeWWnp1v6ORCUyfnG3p2vqlON9drX7RbN10/pT1jtaGsJwBcqsoH/FqtTq4fPjanI8fO6pdPz+uXT89rbqHZfz2JTJ38wt9JZFKv+SXTY3rddVN6/fVT+vMDkzIrpmM227ka5WOrm+tFUyN6+dUTGq8nQ+8XgGp6voCvXPLUkkiHX7FPh1+xr992cqGhXz49r189Pa+lZkdX7apr30SmqyYyXbWrrplddblLjx4/p0eeOKcjx87qv397Uvc9cnyg97x+76heefWEXrF/l165f0IHdo9oV5ZoV5ZqvJ6olmx+KKTdzdXs5Ormrl31RFFkm36fu+vMUkuPn17SH+aW9OSZZe2bzHTTdVN62b5dip/j5wCErXIV/J+Ku+vY6WX95sS8IjPV01hZEhWPaaQkMj1+alm/fmZevz4xr18/s6DHTy9ps193PYm0K0uUxsVNShrtbj/Ye8ykyZFUu0dS7R6tafdoqtFarKfONfTHuUXNNzqbrudYLdaN1+7WTddN6XXX7da+iay/t7H2vVyusVqi8SzRrnqqsXqs8SzReD3RSBrL7E+/kXB3rbS7yl3s5QBbxBDNFWK51dFvTixobqGphUZHi4128djsaL7RUaebK0tj1ZOo/1hPI0Vmml9p6+xyW+dW2jq33NK55baWmh1ds3tEB6dH9eLpcb1kekwvnh7TgakRPX1uRY88cVaPHDunnz95Vr9+ZmHdBuNSRCaN1RKN1mON1YvQH60VB6hzl/Lclbur60Vox5EpjSPV4ki1JFIaF8/b3Vxnl9s6v9zW2bIPrW5xItrMrrpeOjOmG64a10tniq9rdmc6Od/Uk2eX9eSZFT15dlnHz67omXMrSuKo3Asq9oR2ZYkmslRZGvffL4lNtbjY2NbTWCNprCyNNVIrlkfSWElsanWKIbViaG11iK3YCHbXbQhbnVwTI2l/L29fuZe3ezRdtxF0d3VyV6frana6Wmp1tdQsPuvlZrd4bHW01OpqpdXRUrOrlXbxPc1Orr3jNe2fyHT15Iiunsy0fzLT9HhdcWTKcy/Wr1yfVjdXEplGa7FGa8nz7rG5u5qdXAuNjp46t6Lj5e+09/jsfFP7Juq6YWZcN1y1+rV7tNbfUzx2ZllPnF7WE2eWdez0spaaHdXTqPh7TWLVktXlLI00UouVJbHq6erftZnJ3dX/iywX0jhSlhY/W0+j/s9JxR5tu+vqlI/tbtH35VZXK62uVtodrbRyLbc6cpf2jNW0d7ym6fG69o7XtGesploclXu7y3r81JKOnV7SH08v68kzy5oaTYu/vfJv8IarxnfE8bZtCXgzyyT9p6RrJT0q6f3+PG9WhYDfTsutjn5x/LzOr7RV72081mxIJGmp2dVCs62lZleLzbYWm10tNjpaana01Cofm10ttYqQkhXhH5kpjkxmpsikbl7+5+us+Y/YyZXEVux9jKSaGq1p91iq3SM1uVx/nFvS7+cW9djJzfdG4si0fzLTtVOj2r87U5675hsdLZQbyYVGR/ONtpqdXO1uvume0uUyKwKotcnZ0bUkUj2O1M5zdbq+6XGcixlJY43WioA8vdjqb/x64shk0kX/7XoSaazcCKdxpEa72Hg02t3nvDbT5EiqF02NaN9EpmfON/SHucV1Z4FPj9fUaOdabK7/bK6eyDQxkqjVKTY4zU6uZrkx3MrvYNh6t/LsiUw6MDWi6/aM6sxS+4J+T42mmhxJ1XVXnkudPFc3l7p5rufq3mYbrt5zkxRF1v9/01t+/18d1D/ecsOW+rRdY/B3Sjru7reb2f2SbpX03SG+H57HaC3RX75k73avxkW5u04vtfTYyUWdON/QVRP1ItQnMyXx4Kdt9DYy7W5eVtF5P+hW2mXF1+qq3c3LirMI1l71WduwAawnxZ6BmWml1dXJhWJ21drZVs1OvmbvIVIamdKk2JMZr6/uAY3VivAdqycaqxdV90gar6u8e9XyM+cbOnG+oRPzxWPu3l/PWry6rt282IgvNbvlnkGxEW518/6eS5ZGGklj1dNYY7VYB6ZG9aKpER2YGtFEll7w+3vq7Ioem1vQ755d1B/mlpSlka7bO6br94zq+r2junbPaP/OaZvpdHM1yt97b+PS2xOSig2mVIReT7vc41k7YaF3V7Y0LvYGkyjqL9eSSKO1on+93+NILZaZdHappVOLLZ1abOr0YkunF5s6v9LW1ZOZXjw9poPTY7p2anTdMbA8dz11bqVfbPx+bklLzY7iqChiYjPFcfEYmZ536HK1f9Z/nrvLvXjs5q683Ot9yfTYRf+mt2KYFfxXJd3n7veZ2Yclzbj7x57r+6ngAeDSPV8FP8wzWfdKOl8uz0vas8mK3WNms2Y2Ozc3N8RVAYDqGWbAn5I0WS5Pls/Xcfd73f2Qux+amZkZ4qoAQPUMM+AflHRbuXxY0sNDfC8AwAbDDPivSDpgZo9KOqMi8AEAL5ChzaJx96ak24f17wMAnl+wlwsGgKoj4AEgUAQ8AATqirkWjZnNSTq2xR+f1ibTMCuiqn2n39VCv5/b9e6+6TzzKybgL4eZzT7XmVyhq2rf6Xe10O+tYYgGAAJFwANAoEIJ+Hu3ewW2UVX7Tr+rhX5vQRBj8ACAC4VSwQMANiDgASBQOzrgzSwzs/vN7KiZfdmGcWfoK4yZpWb2zXK5Mv03sy+Z2U/M7BtmNl6FfptZYmZfM7MfmdkXq/R5S5KZfdjMvmdm02b2AzP7hZl9crvXa1jM7A1mdtzMflh+3Xi5n/eODnit3hbwRklTKm4LGCwzG5F0RKv9rET/zexmSYm7v1HShKS7VYF+S3qXpKPu/mZJ+yV9QNXot8zsekl3lU8/JOlbkm6U9DYze9m2rdhwTUn6vLvf7O43S3qDLvPz3ukBf1jSA+XyQ5Ju2cZ1GTp3X3H310o6XjZVpf/PSvpMuRxJ+riq0e9vS/q0mSWSdku6SdXot1R83r1bfB6W9IC755K+r3D7PSXp78zsZ2Z2n6S36jI/750e8Be9LWDgKtF/d/+du//MzN4tKZf0c1Wj34vuvizpRyo2cpX4vM3sDklHJf2qbKpEvyU9Jumf3f0vVOyxvUeX2e+dHvAXvS1g4CrTfzN7p6QPSnqHpBOqQL/NbK+Z1SW9SUV19xpVoN8q7iPxVkn/Ien1Kq7HUoV+Py7pe2uWc11mv3d6wFf9toCV6L+ZXS3po5Jud/cFVaTfkj4i6b3u3pW0LOkTqkC/3f2Ocgz6fSqOOX1O0m1mFkl6iwLtt6QPS3pf2c/XqPj8L+vz3ukBX/XbAlal/3ep2GX9jpn9UFKqavT7c5LuNrMfSzot6QuqRr83+qykt0t6VNK33P2xbV6fYfk3Sf8g6aeSvq4/wefNmawAEKidXsEDAJ4DAQ8AgSLgASBQBDwABIqAB4BAEfAAEKj/BwNgNJexqkxaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
